# -*- coding: utf-8 -*-

from scrapy import Spider, Item, Field, log ,Request
from webspider.items import CarBrandItem, CarSerialItem, CarModelItem
import json

DUMP_DATA = False

def output(text):
    if DUMP_DATA:
        print text

DRIVE_CHOICES_OLD = (
    (0,u'未知'),
    (1,u'前置前驱'),
    (2,u'前置后驱'),
    (3,u'中置四驱'),
    (4,u'前置四驱'),
    (5,u'中置后驱'),
    (6,u'混动'),
    (7,u'后置后驱'),
    (8,u'电动'),
    (9,u'后置四驱'),
)

TRANSMISSION_CHOICES_OLD = (
    (0,u'未知'),
    (1,u'2挡自动'),
    (2,u'4挡自动'),
    (3,u'5挡自动'),
    (4,u'6挡自动'),
    (5,u'7挡自动'),
    (6,u'8挡自动'),

    (10,u'4挡手动'),
    (11,u'5挡手动'),
    (12,u'6挡手动'),

    (20,u'4挡手自一体'),
    (21,u'5挡手自一体'),
    (22,u'6挡手自一体'),
    (23,u'7挡手自一体'),
    (24,u'8挡手自一体'),
    (25,u'9挡手自一体'),

    (30,u'5挡AMT'),
    (31,u'6挡AMT'),
    (32,u'6挡序列'),

    (40,u'6挡双离合'),
    (41,u'7挡双离合'),
    (42,u'8挡双离合'),

    (50,u'CVT无级变速'),
    (51,u'CVT无级变速(模拟6挡)'),
    (52,u'CVT无级变速(模拟7挡)'),
    (53,u'CVT无级变速(模拟8挡)'),
    (54,u'CVT无级变速模拟7挡'),
    (55,u'CVT无级变速模拟8挡'),

    (60,u'7挡ISR变速箱'),

    (70,u'电动车单速变速箱'),

)

TYPE_CHOICES_OLD = (
    (0,u'未知'),
    (1,u'微型车'),
    (2,u'小型车'),
    (3,u'紧凑型车'),
    (4,u'中型车'),
    (5,u'中大型车'),
    (6,u'豪华车'),
    (7,u'跑车'),
    (8,u'MPV'),
    (9,u'小型SUV'),
    (10,u'紧凑型SUV'),
    (11,u'中型SUV'),
    (12,u'中大型SUV'),
    (13,u'全尺寸SUV'),
    (14,u'微面'),
    (15,u'微卡'),
    (16,u'轻客'),
    (17,u'皮卡'),
)

TRANSMISSION_CHOICES = {}
for c in TRANSMISSION_CHOICES_OLD:
    l = list(c)
    TRANSMISSION_CHOICES[l[1]] = l[0]

DRIVE_CHOICES = {}
for c in DRIVE_CHOICES_OLD:
    l = list(c)
    DRIVE_CHOICES[l[1]] = l[0]

TYPE_CHOICES = {}
for c in TYPE_CHOICES_OLD:
    l = list(c)
    TYPE_CHOICES[l[1]] = l[0]

class CarSQLSpider(Spider):
    name, start_urls = 'CarSpider', ['http://car.autohome.com.cn/zhaoche/pinpai/?pvareaid=101451']

    def parse(self, response):
        for car_brand_html in response.css("div.uibox-con dl"):

            # Car Brand
            car_brand_name = car_brand_html.css("dl dt p a::text").extract()[0].encode('utf8')
            car_brand_id = car_brand_html.css("dl dt p a::attr(href)").extract()[0].encode('utf8').replace("http://car.autohome.com.cn/pic/brand-", "").replace(".html", "")
            car_brand_id = car_brand_id.replace("http://car.autohome.com.cn/price/brand-", "").replace(".html", "")

            # output("INSERT INTO base_carbrand (id, name, external_id) VALUES (%s, '%s', %s) ON DUPLICATE KEY UPDATE name='%s';" % (car_brand_id, car_brand_name, car_brand_id, car_brand_name))

            car_brand_model = CarBrandItem().django_model

            try:
                obj_car_brand = car_brand_model.objects.get(external_id=car_brand_id)
            except car_brand_model.DoesNotExist:
                obj_car_brand = car_brand_model()

            obj_car_brand.name = car_brand_name
            obj_car_brand.external_id = car_brand_id
            obj_car_brand.save()

            # Car Serial
            for car_serial_html in car_brand_html.css("dd ul li h4"):
                car_serial_name = car_serial_html.css("h4 a::text").extract()
                if len(car_serial_name) > 0:
                    car_serial_name = car_serial_name[0].encode('utf8')

                car_serial_id = car_serial_html.css("h4 a::attr(href)").extract()

                if len(car_serial_id) > 0:
                    car_serial_id = car_serial_id[0].encode('utf8').replace("http://www.autohome.com.cn/", "").replace("/", "")

                car_serials_model = CarSerialItem().django_model

                try:
                    obj_car_serial = car_serials_model.objects.get(external_id=car_serial_id)
                except car_serials_model.DoesNotExist:
                    obj_car_serial = car_serials_model()

                obj_car_serial.name = car_serial_name
                obj_car_serial.external_id = car_serial_id
                obj_car_serial.brand = obj_car_brand
                obj_car_serial.save()
                # output("INSERT INTO base_carserial (id,name, brand_id, external_id) VALUES (%s,'%s', %s, %s) ON DUPLICATE KEY UPDATE name='%s', brand_id=%s;" % (car_serial_id,car_serial_name, car_brand_id, car_serial_id, car_serial_name, car_brand_id))

                yield Request(url='http://www.autohome.com.cn/' + car_serial_id + '/', meta={'parent': obj_car_serial}, callback=self.parseMore)

    def parseMore(self, response):
        #print u"========= On Sale Serial ========="

        # Car Model
        for car_model_html in response.css("li.grey-bg"):
            car_model_name = car_model_html.css("div.interval01-list-cars-infor p:first-child a::text").extract()[0].encode('utf8')
            car_model_id = car_model_html.css("div.interval01-list-cars-infor p:first-child a::attr(href)").extract()[0].encode('utf8').replace("/spec/","").replace("/#pvareaid=101605", "")
            car_model_value = car_model_html.css("div.interval01-list-guidance div::text").extract()[0].encode('utf8').lstrip().replace("万","")

            car_model_drive_type = ''
            car_model_drive_type_array = car_model_html.css("div.interval01-list-cars-infor p span:first-child::text").extract()
            if len(car_model_drive_type_array) > 0:
                car_model_drive_type = car_model_drive_type_array[0].encode('utf8')
                if car_model_drive_type in ["电动","混动"]:
                    try:
                        car_model_drive_type = car_model_html.css("div.interval01-list-cars-infor p span:nth-of-type(2)::text").extract()[0].encode('utf8')
                    except:
                        pass
            car_model_transmission = ''
            car_model_transmission_array = car_model_html.css("div.interval01-list-cars-infor p span:last-child::text").extract()
            if len(car_model_drive_type_array) > 0:
                try:
                    car_model_transmission = car_model_transmission_array[0].encode('utf8')
                except:
                    pass


            if car_model_name not in ["补贴","减税"]:
                car_model_drive_type = DRIVE_CHOICES.get(car_model_drive_type.decode('utf-8'), 0)
                car_model_transmission = TRANSMISSION_CHOICES.get(car_model_transmission.decode('utf8'), 0)

            car_model = CarModelItem().django_model

            try:
                obj_car_model = car_model.objects.get(external_id=car_model_id)
            except car_model.DoesNotExist:
                obj_car_model = car_model()

            obj_car_model.name = car_model_name
            obj_car_model.external_id = car_model_id
            obj_car_model.serial = response.meta['parent']
            obj_car_model.transmission = car_model_transmission
            obj_car_model.save()


            # output("INSERT INTO base_carmodel (id,transmission,drive_type,name, price,serial_id, external_id) VALUES (%s,%s,%s,'%s', %s, %s, %s) ON DUPLICATE KEY UPDATE name='%s', serial_id=%s;" % (car_model_id,car_model_transmission,car_model_drive_type,car_model_name, car_model_value,response.meta["parent"], car_model_id, car_model_name, response.meta["parent"]))
        yield Request(url= response.url + 'sale.html', meta={'parent': response.meta["parent"]}, callback=self.parseMoreAndMore)

        # for car_offsale_model_html in response.css("div#drop2 li"):
        #     car_offsale_model_id = car_offsale_model_html.css("li a::attr(data)").extract()[0].encode('utf8')
        #     url = 'http://www.autohome.com.cn/ashx/series_allspec.ashx?s=%s&y=%s' % (response.meta["parent"].id, car_offsale_model_id)
        #     print url + "==========================="
        #     yield Request(url= url, meta={'parent': response.meta["parent"]}, callback=self.parseOffSaleJson)


    def parseMoreAndMore(self,response):
        #print u"========= Off Sale Serial ========="

        car_model_transmission = TRANSMISSION_CHOICES.get('未知', 0)

        # Car Model
        for car_model_html in response.css("div.modtab1 table tr"):
            car_model_name = car_model_html.css("td.name_d a::attr(title)").extract()[0].encode('utf8')
            car_model_id = car_model_html.css("td.name_d a::attr(href)").extract()[0].encode('utf8').replace("spec/","").replace("/", "")
            car_model_price = car_model_html.css("td.price_d::text").extract()[0].encode('utf8')

            # print(car_model_name, car_model_id, car_model_price)

            if car_model_name not in ["补贴","减税"]:
                car_model = CarModelItem().django_model

                try:
                    obj_car_model = car_model.objects.get(external_id=car_model_id)
                except car_model.DoesNotExist:
                    obj_car_model = car_model()

                obj_car_model.name = car_model_name
                obj_car_model.external_id = car_model_id
                obj_car_model.serial = response.meta['parent']
                obj_car_model.transmission = car_model_transmission
                obj_car_model.price = car_model_price.replace("万","")
                r = obj_car_model.save()

                print obj_car_model.id

                print obj_car_model.name, obj_car_model.external_id, obj_car_model.serial.id, obj_car_model.transmission, obj_car_model.price

                # output("INSERT INTO base_carmodel (id,name, serial_id, external_id) VALUES (%s,'%s', %s, %s) ON DUPLICATE KEY UPDATE name='%s', serial_id=%s;" % (car_model_id,car_model_name, response.meta["parent"], car_model_id, car_model_name, response.meta["parent"]))
                # output("INSERT INTO base_carmodel (id,transmission,drive_type,name, price,serial_id, external_id) VALUES (%s,1,0,'%s', 0, %s, %s) ON DUPLICATE KEY UPDATE name='%s', serial_id=%s;" % (car_model_id,car_model_name,response.meta["parent"], car_model_id, car_model_name, response.meta["parent"]))

    # def parseOffSaleJson(self,response):
    #     data = json.loads(unicode(response.body, 'gbk'))
    #
    #     for car in data['Spec']:
    #         print car['Name']
    #         if car['Price'] == u"<span>暂无</span>":
    #             car['Price'] = 0.00
    #         else:
    #             car['Price'] = car['Price'].encode('utf8').replace("万","")
    #
    #         car_model_id = car['Id']
    #         car_model_name = car['Name'].encode('utf8')
    #         car_model_value = car['Price']
    #         # car_model_drive_type = car['DrivingModeName']
    #         # car_model_transmission = car['Transmission']
    #         car_model_drive_type = DRIVE_CHOICES.get(car['DrivingModeName'], 0)
    #         car_model_transmission = TRANSMISSION_CHOICES.get(car['Transmission'], 0)
    #
    #         car_model = CarModelItem().django_model
    #
    #         try:
    #             obj_car_model = car_model.objects.get(external_id=car_model_id)
    #             print "off-sale car created******************"
    #         except car_model.DoesNotExist:
    #             print "off-sale car existed=================="
    #             obj_car_model = car_model()
    #
    #         obj_car_model.name = car_model_name
    #         obj_car_model.external_id = car_model_id
    #         obj_car_model.serial = response.meta['parent']
    #         obj_car_model.transmission = car_model_transmission
    #         obj_car_model.save()
    #         print "off-sale car model saved~~~~~~~~~~~~~~~~~~~~"
    #         #car_model_name = car_model_html["Spec"]["Name"]
    #         # print "=======================",car_model_value
    #         # output("INSERT INTO base_carmodel (id,transmission,drive_type,name, price,serial_id, external_id) VALUES (%s,%d,%d,'%s', %s, %s, %s) ON DUPLICATE KEY UPDATE name='%s', serial_id=%s;" % (car_model_id,car_model_transmission,car_model_drive_type,car_model_name, car_model_value,response.meta["parent"], car_model_id, car_model_name, response.meta["parent"]))
    #         # output("INSERT INTO base_carmodel (id,transmission,drive_type,name, price,serial_id, external_id) VALUES (%s,%s,%s,'%s', %s, %s, %s) ON DUPLICATE KEY UPDATE name='%s', serial_id=%s;" % (car_model_id,car_model_transmission,car_model_drive_type,car_model_name, car_model_value,response.meta["parent"], car_model_id, car_model_name, response.meta["parent"]))


class CarSerialType(Spider):
    name, start_urls = 'CarSerialType', ['http://car.autohome.com.cn/zhaoche/jibie/']

    def parse(self,response):
        # print "==================="

        for i in range(1,16):
            for car_serial_html in response.css("div.uibox div#tab%d" % i):
                car_serial_type = car_serial_html.css("div#tab%d" % i  + " a::text").extract()[0].encode('utf8')
                car_serial_name = car_serial_html.css("div#tab%d" % i + " + div.uibox-con dl dd ul li h4 a::text").extract()
                # print "=================================================" + car_serial_type
                # print "==== BEFORE CAR TYPE ====", car_serial_type
                car_serial_type = TYPE_CHOICES.get(car_serial_type.decode('utf8'))
                # print "==== AFTER CAR TYPE ====", car_serial_type
                for s in car_serial_name:
                    # print "=======================" + s.encode('utf-8')
                    output("UPDATE base_carserial SET car_type = %d WHERE name = '%s';" % (car_serial_type,s.encode('utf-8')))

#for i in range(65,90):
#      print chr(i)


# class CarBrandType(Spider):
#     name, start_urls = 'CarBrandType', ['http://car.autohome.com.cn/zhaoche/pinpai/']

#     def parse(self,response):
#         print "==========="
#         for car_brand_html in response.css("div.uibox div.uibox-con dl"):
#             car_brand_img = car_brand_html.css("dl a.pic img::attr(src)").extract()[0].encode('utf8')
#             car_brand_name = car_brand_html.css("dl dt p:last-child a::text").extract()[0].encode('utf8')
#             print car_brand_img
#             print car_brand_name
#             # output("UPDATE base_carbrand SET img = '%s' WHERE name = '%s';" % (car_brand_img,car_brand_name))
#             # print "UPDATE base_carbrand SET img = '%s' WHERE name = '%s';" % (car_brand_img,car_brand_img)


class CarBrandImg(Spider):
    name, start_urls = 'CarBrandImg', ['http://car.autohome.com.cn/zhaoche/pinpai/']

    def parse(self,response):
        for x in range(65,91):
            for div_block_html in response.css("div.uibox div#tab%s" % chr(x)):
                car_brand_py = div_block_html.css("div#tab%s" % chr(x) + "::text").extract()[0].encode('utf8')
                car_brand_div_html = div_block_html.css("div#tab%s" % chr(x) + "+ div.uibox-con dl")

                for car_brand_html in car_brand_div_html:
                    car_brand_img = car_brand_html.css("dt a.pic img::attr(src)").extract()[0].encode('utf8')
                    car_brand_name = car_brand_html.css("dt p:last-child a::text").extract()[0].encode('utf8')
                    output("UPDATE base_carbrand SET letter = '%s' , logo = '%s' WHERE name = '%s';" % (car_brand_py,car_brand_img,car_brand_name))
